mongodb存储图片和文件实践

2024-07-14 03:28| 来源: 网络整理| 查看: 265

可能很多人会问为何要用mongodb存储图片和文件，我的理由是mongodb作存储方便稳定易扩展，我们自己写的文件存储服务器肯定很LOW，除了含有巨量图片的大型系统，一般由公司自研图片存储外，大部分情况下，我们用mongodb足以应付。另外，很多人根本就不知道或者没尝试过用mongodb存储文件，而对方还故意问我为什么用这个。本文特此奉上代码并提供MongoDBClient与spring的集成。

主要依赖：

org.imgscalr imgscalr-lib 4.2 org.mongodb mongo-java-driver 3.0.0

public class MongoDBConfig { private String addresses; private List credentials; public String getAddresses() { return addresses; } public void setAddresses(String addresses) { this.addresses = addresses; } public List getCredentials() { return credentials; } public void setCredentials(List credentials) { this.credentials = credentials; } public List buildCredentials() { List mongoCredentials = new ArrayList(); for(MongoDBCredential item : this.credentials) { MongoCredential credential = MongoCredential.createCredential(item.getUsername(),item.getDatabaseName(),item.getPassword().toCharArray()); mongoCredentials.add(credential); } return mongoCredentials; } public List buildAddresses() throws Exception{ List serverAddresses = new ArrayList(); String[] sources = addresses.split(";"); for(String item : sources) { String[] hp = item.split(":"); serverAddresses.add(new ServerAddress(hp[0],Integer.valueOf(hp[1]))); } return serverAddresses; } }

public class MongoDBCredential { private String databaseName; private String username; private String password; public String getDatabaseName() { return databaseName; } public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } public String getUsername() { return username; } public void setUsername(String username) { this.username = username; } public String getPassword() { return password; } public void setPassword(String password) { this.password = password; } }

public class MongoDBDriver { private Properties properties; /** */ private MongoClient mongo = null; /** */ private Integer connectionsPerHost = 32; private Integer threadsAllowedToBlockForConnectionMultiplier = 5; private Integer maxWaitTime = 30000; private Integer connectTimeout = 30000; private Integer socketTimeout = 30000; private Integer maxConnectionIdle = 6000; private MongoDBConfig configuration; public void setProperties(Properties properties) { this.properties = properties; } public void setConfiguration(MongoDBConfig configuration) { this.configuration = configuration; } public void init() throws Exception{ if(properties != null){ String perHost = properties.getProperty("mongodb.driver.connectionsPerHost"); if(StringUtils.isNotBlank(perHost)){ connectionsPerHost = Integer.valueOf(perHost); } String multiplier = properties.getProperty("mongodb.driver.threadsAllowedToBlockForConnectionMultiplier"); if(StringUtils.isNotBlank(multiplier)){ threadsAllowedToBlockForConnectionMultiplier = Integer.valueOf(multiplier); } / String waitTime = properties.getProperty("mongodb.driver.maxWaitTime"); if(StringUtils.isNotBlank(waitTime)){ maxWaitTime = Integer.valueOf(waitTime); } String ctimeout = properties.getProperty("mongodb.driver.connectTimeout"); if(StringUtils.isNotBlank(ctimeout)){ connectTimeout = Integer.valueOf(ctimeout); } String stimeout = properties.getProperty("mongodb.driver.socketTimeout"); if(StringUtils.isNotBlank(stimeout)){ socketTimeout = Integer.valueOf(stimeout); } String mci = properties.getProperty("mongodb.driver.maxConnectionIdle"); if(StringUtils.isNotBlank(mci)){ maxConnectionIdle = Integer.valueOf(mci); } } init,db check and connected. execute(); } private void execute() throws Exception{ MongoClientOptions.Builder builder = MongoClientOptions.builder(); builder.connectionsPerHost(this.connectionsPerHost); builder.threadsAllowedToBlockForConnectionMultiplier(threadsAllowedToBlockForConnectionMultiplier); builder.socketKeepAlive(true); builder.maxWaitTime(this.maxWaitTime); builder.connectTimeout(this.connectTimeout); builder.socketTimeout(this.socketTimeout); builder.maxConnectionIdleTime(maxConnectionIdle); MongoClientOptions options = builder.build(); this.mongo = new MongoClient(configuration.buildAddresses(), configuration.buildCredentials(),options); } public void close() { mongo.close(); } public Mongo getMongo() { return mongo; } public MongoDatabase getDatabase(String dbName) { return mongo.getDatabase(dbName); } /** * old api * @param dbName * @return */ public DB getDB(String dbName) { return mongo.getDB(dbName); } }

public class ObjectIdCreator{ public static String creator(){ return new ObjectId().toString(); } }

public class CustomDateDeserializer extends JsonDeserializer { private static final String[] pattern = { "yyyy-MM-dd HH:mm:ss" }; @Override public Date deserialize(JsonParser arg0, DeserializationContext arg1) throws IOException, JsonProcessingException { if (JsonToken.VALUE_STRING.equals(arg0.getCurrentToken())) { try { return DateUtils.parseDate(arg0.getText(), pattern); } catch (ParseException e) { e.printStackTrace(); } } return null; } }

public class CustomDateSerializer extends JsonSerializer { private static final String pattern = "yyyy-MM-dd HH:mm:ss"; @Override public void serialize(Date value, JsonGenerator jgen, SerializerProvider provider) throws IOException, JsonProcessingException { jgen.writeString(DateFormatUtils.format(value, pattern)); } }

public class MongoDBClient { protected MongoDBDriver mongoDBDriver; protected String databaseName; public void setMongoDBDriver(MongoDBDriver mongoDBDriver) { this.mongoDBDriver = mongoDBDriver; } public void setDatabaseName(String databaseName) { this.databaseName = databaseName; } public MongoCollection getCollection(String collectionName) { MongoDatabase db = mongoDBDriver.getDatabase(this.databaseName); return db.getCollection(collectionName); } public MongoDatabase getDatabase() { return mongoDBDriver.getDatabase(this.databaseName); } public static void main(String[] args) throws Exception{ MongoDBDriver mongoDBDriver = new MongoDBDriver(); try{ MongoDBConfig mongoDBConfig = new MongoDBConfig(); //mongoDBConfig.setAddresses("61.171.123.234:27017"); mongoDBConfig.setAddresses("61.171.123.234:27017"); List credentials = new ArrayList(); MongoDBCredential credential = new MongoDBCredential(); credential.setDatabaseName("whatsmars-common"); credential.setUsername("whatsmars"); //credential.setPassword("haodai.com"); credential.setPassword("passwordiscommon"); credentials.add(credential); mongoDBConfig.setCredentials(credentials); mongoDBDriver.setConfiguration(mongoDBConfig); mongoDBDriver.init(); MongoDBClient client = new MongoDBClient(); client.setDatabaseName("whatsmars-common"); client.setMongoDBDriver(mongoDBDriver); ListCollectionsIterable documents = client.getDatabase().listCollections(); MongoCursor it = documents.iterator(); while (it.hasNext()) { Document item = it.next(); System.out.println(item.toJson()); } it.close(); } catch (Exception e) { e.printStackTrace(); } finally { mongoDBDriver.close(); } } }

public class GridFSClient extends MongoDBClient{ private GridFS _gridFS = null; private Object lock = new Object(); protected static final String[] IMAGE_FORMAT = {"jpg","jpeg","png"}; public void setMongoDBDriver(MongoDBDriver mongoDBDriver) { this.mongoDBDriver = mongoDBDriver; } public GridFS getInstance() { if(_gridFS != null) { return _gridFS; } synchronized (lock) { if(_gridFS != null) { return _gridFS; } _gridFS = new GridFS(mongoDBDriver.getDB(this.databaseName)); return _gridFS; } } public void close() { mongoDBDriver.close(); } /** * * @param inputStream 文件流 * @param format 文件格式，“pdf”，“png”等，不包含后缀符号“.” * @return */ public String saveFile(InputStream inputStream,String format,String uid) { try { GridFS gridFS = getInstance(); //随机生成文件名称，多次重试 String filename = this.randomFileName(); //如果有文件重复，则重新生成filename while (true) { GridFSDBFile _current = gridFS.findOne(filename); //如果文件不存在，则保存操作 if (_current == null) { break; } filename = this.randomFileName(); } GridFSInputFile file = gridFS.createFile(inputStream, filename); if(format != null) { file.put("format", format); } if(uid != null) { file.put("uid",uid); } file.put("content-type","application/octet-stream"); file.save(); return concat(filename,format); }catch (Exception e) { throw new RuntimeException(e); } finally { try{ inputStream.close(); }catch (Exception ex) { // } } } private String concat(String filename,String format) { if(format == null) { return filename; } if(format.startsWith(".")) { return filename + format; } return filename + "." + format; } private String randomFileName() { return RandomStringUtils.random(32, true, true).toLowerCase(); } public void delete(String filename) { GridFS gridFS = getInstance(); gridFS.remove(filename); } public InputStream getFile(String filename) { GridFS gridFS = getInstance(); GridFSDBFile _current = gridFS.findOne(filename); if(_current == null) { return null; } return _current.getInputStream(); } public InputStream getImage(String filename,String path) throws Exception{ //获取最大边,等比缩放 if(ImageSizeEnum.valueOfPath(path) == null) { return null; } GridFS gridFS = getInstance(); GridFSDBFile _current = gridFS.findOne(filename); if(_current == null) { return null; } int size = ImageSizeEnum.valueOfPath(path).size; int max = (Integer)_current.get("max");//图片的实际尺寸 InputStream result = null; //裁剪 if(size < max) { InputStream inputStream = _current.getInputStream(); BufferedImage image = ImageIO.read(inputStream); inputStream.close(); ByteArrayOutputStream bos = new ByteArrayOutputStream(); BufferedImage thumbnail = Scalr.resize(image, size);//保留最大尺寸 String format = (String) _current.get("format"); ImageIO.write(thumbnail, format, bos); result = new ByteArrayInputStream(bos.toByteArray()); } else { result = _current.getInputStream(); } return result; } /** * * @param inputStream 输入流 * @return * @throws Exception */ public String saveImage(InputStream inputStream,String uid) throws Exception{ BundleEntry bundleEntry = this.drain(inputStream); if(bundleEntry == null) { throw new RuntimeException("file isn't a image!"); } ByteArrayInputStream bis = bundleEntry.inputStream; String _currentFileName = this.isExistedImage(bundleEntry); //如果文件md5已存在 if(_currentFileName != null) { return _currentFileName; } String format = bundleEntry.format; GridFS gridFS = getInstance(); String filename = this.randomFileName(); //检测文件名称 while(true){ GridFSDBFile _current = gridFS.findOne(filename); //如果文件不存在，则保存操作 if (_current == null) { break; } //否则，重新生成文件名称 filename = randomFileName(); } //图片处理 bis.reset(); //保存原图 GridFSInputFile _inputFile = gridFS.createFile(bis, filename); if(uid != null) { _inputFile.put("uid", uid); } _inputFile.put("max",bundleEntry.max); _inputFile.put("crc",bundleEntry.crc); _inputFile.put("format",format); _inputFile.put("md5_source",bundleEntry.md5); _inputFile.save(); return concat(filename,format); } private String isExistedImage(BundleEntry entry) { GridFS gridFS = getInstance(); DBObject query = new BasicDBObject(); query.put("crc",entry.crc); query.put("md5_source",entry.md5); GridFSDBFile _current = gridFS.findOne(query); //根据MD5值查询，检测是否存在 if(_current == null) { return null; } String format = (String)_current.get("format"); if(format.startsWith(".")) { return _current.getFilename() + format; } return _current.getFilename() + "." + format; } /** * 因为图片的stream需要reset，所以需要将流全部汲取 * @param inputStream * @return * @throws Exception */ protected BundleEntry drain(InputStream inputStream) throws Exception{ // ByteArrayOutputStream bos = new ByteArrayOutputStream(); //计算源文件的md5、crc，以防止图片的重复上传 Adler32 crc = new Adler32(); try{ while(true) { int _c = inputStream.read(); if(_c == -1) { break; } bos.write(_c); crc.update(_c); } }catch (Exception e) { throw new RuntimeException(e); } finally { inputStream.close(); } //第一步：图片格式 List formats = new ArrayList();// ImageInputStream imageInputStream = ImageIO.createImageInputStream(new ByteArrayInputStream(bos.toByteArray())); imageInputStream.mark(); try { Iterator it = ImageIO.getImageReaders(imageInputStream); while (it.hasNext()) { ImageReader reader = it.next(); String format = reader.getFormatName().toLowerCase(); if(ArrayUtils.contains(IMAGE_FORMAT, format)) { formats.add(format); } } }catch (Exception ex) { // } //如果格式不合法，则直接返回 if(formats.isEmpty()) { try { imageInputStream.close(); } catch (Exception e) { // } return null; } String md5 = DigestUtils.md5Hex(bos.toByteArray());//求原始图片的MD5，和crc System.out.println("md5:" + md5); imageInputStream.reset(); BufferedImage image = ImageIO.read(imageInputStream); //获取最大边,等比缩放 int max = Math.max(image.getHeight(), image.getWidth()); bos = new ByteArrayOutputStream(); //如果尺寸超过最大值，则resize if(max > ImageSizeEnum.PIXELS_MAX.size) { max = ImageSizeEnum.PIXELS_MAX.size; } String format = formats.get(0); BufferedImage thumbnail = Scalr.resize(image, max);//保留最大尺寸 ImageIO.write(thumbnail, format, bos); return new BundleEntry(new ByteArrayInputStream(bos.toByteArray()),md5,crc.getValue(),format,max); } protected class BundleEntry { String md5; long crc; String format; int max; ByteArrayInputStream inputStream; BundleEntry(ByteArrayInputStream inputStream,String md5,long crc,String format,int max) { this.md5 = md5; this.crc = crc; this.inputStream = inputStream; this.format = format; this.max = max; } } public static void main(String[] args) throws Exception{ MongoDBDriver mongoDBDriver = new MongoDBDriver(); try { GridFSClient client = new GridFSClient(); MongoDBConfig mongoDBConfig = new MongoDBConfig(); //mongoDBConfig.setAddresses("61.172.238.149:27017"); mongoDBConfig.setAddresses("61.172.240.58:27017"); List credentials = new ArrayList(); MongoDBCredential credential = new MongoDBCredential(); credential.setDatabaseName("whatsmars-fs"); credential.setUsername("whatsmars"); //credential.setPassword("haodai.com"); credential.setPassword("passwordisfs"); credentials.add(credential); mongoDBConfig.setCredentials(credentials); mongoDBDriver.setConfiguration(mongoDBConfig); mongoDBDriver.init(); client.setDatabaseName("whatsmars-fs"); client.setMongoDBDriver(mongoDBDriver); testUpload(client); //testClear(client.getInstance()); //testGetImage(client.getInstance(),"xhgcguccxumuyl9hzdombgfvzgriv7rf",null); } catch (Exception e) { e.printStackTrace(); } finally { mongoDBDriver.close(); } } protected static void testClear(GridFS gridFS) { DBCursor cursor = gridFS.getFileList(); while (cursor.hasNext()) { DBObject dbObject = cursor.next(); String filename = (String)cursor.next().get("filename"); System.out.println(filename); System.out.println(dbObject.toString()); gridFS.remove(filename); } cursor.close(); } protected static void testUpload(GridFSClient client) throws Exception{ FileInputStream inputStream = new FileInputStream(new File("/data/tmp/222222222.jpg")); try { String filename = client.saveImage(inputStream, null); System.out.println(filename); String source = filename.substring(0,filename.lastIndexOf(".")); System.out.println(source); InputStream result = client.getImage(source, "x4"); if(result == null) { System.out.println("not found!"); } //vejibw36famkscjyksgke7bugzonnyan FileOutputStream outputStream = new FileOutputStream("/data/tmp/" + filename); while (true) { int i = result.read(); if( i == -1) { break; } outputStream.write(i); } outputStream.flush(); outputStream.close(); result.close(); } catch (Exception e) { e.printStackTrace(); } finally { inputStream.close(); } } protected static void testGetImage(GridFS gridFS,String filename,String path) { DBObject query = new BasicDBObject(); query.put("md5_source","9e131ae4ed7337d4712650229b827725"); GridFSDBFile file = gridFS.findOne(query); if(file != null) { System.out.println(file.getFilename()); } } }

public enum ImageSizeEnum { PIXELS_200(200, "x2"), PIXELS_400(400, "x4"), PIXELS_600(600, "x6"), PIXELS_800(800, "x8"), PIXELS_MAX(1024, "xm");//最大1024 public int size; public String path; ImageSizeEnum(int size, String path) { this.size = size; this.path = path; } public static ImageSizeEnum valueOfPath(String path) { if(path == null) { return null; } for(ImageSizeEnum e : ImageSizeEnum.values()) { if(e.path.equalsIgnoreCase(path)) { return e; } } return null; } }

mongodb-config.properties:

mongodb.driver.connectionsPerHost=32 mongodb.driver.threadsAllowedToBlockForConnectionMultiplier=5 mongodb.driver.maxWaitTime=30000 mongodb.driver.connectTimeout=30000 mongodb.driver.socketTimeout=30000 mongodb.driver.maxConnectionIdle=60000

spring-context.xml:

spring-service.xml:

classpath:mongodb-config.properties

spring-mvc-servlet.xml:

text/plain;charset=UTF-8 text/html;charset=UTF-8

@Controller public class FileController { private static final int SIZE = 1028576;//1M @Autowired private FileService fileService; @RequestMapping("/file_upload") @ResponseBody public String uploadFile(@RequestParam(value = "format",required = false)String format, @RequestParam(value = "uid")String uid, @RequestParam("token")String token, HttpServletRequest request) { String source = DESUtils.decrypt(token, CommonConstants.FS_SECURITY_KEY); if(!source.equals(uid)) { return ResultHelper.renderAsJson(ResultCode.VALIDATE_FAILURE); } ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { InputStream inputStream = request.getInputStream();//body inputstream if (inputStream == null) { return ResultHelper.renderAsJson(ResultCode.PARAMETER_ERROR); } int i = 0; int maxSize = SIZE * 32;//最大32M while (true) { int _c = inputStream.read(); if (_c == -1) { break; } bos.write(_c); i++; if (i > maxSize) { inputStream.close(); bos.close(); return ResultHelper.renderAsJson(ResultCode.VALIDATE_FAILURE, "文件尺寸超过最大限制"); } } inputStream.close(); } catch (Exception e) { return ResultHelper.renderAsJson(ResultCode.SYSTEM_ERROR); } // ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); Result result = this.fileService.uploadFile(bis, format,uid); return ResultHelper.renderAsJson(result); } @RequestMapping("/image_upload") @ResponseBody public String uploadImage(@RequestParam("token")String token, @RequestParam("uid")String uid, @RequestParam(value = "return_size",required = false)Integer returnSize, HttpServletRequest request) { String source = DESUtils.decrypt(token, CommonConstants.FS_SECURITY_KEY); if(!source.equals(uid)) { return ResultHelper.renderAsJson(ResultCode.VALIDATE_FAILURE); } ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { InputStream inputStream = request.getInputStream();//body inputstream if (inputStream == null) { return ResultHelper.renderAsJson(ResultCode.PARAMETER_ERROR); } int i = 0; int maxSize = 6 * SIZE;//最大6M while (true) { int _c = inputStream.read(); if (_c == -1) { break; } bos.write(_c); i++; if (i > maxSize) { inputStream.close(); bos.close(); return ResultHelper.renderAsJson(ResultCode.VALIDATE_FAILURE, "文件尺寸超过最大限制"); } } inputStream.close(); } catch (Exception e) { return ResultHelper.renderAsJson(ResultCode.SYSTEM_ERROR); } // ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray()); Result result = this.fileService.uploadImage(bis,uid,returnSize); return ResultHelper.renderAsJson(result); } @RequestMapping("/ds/{filename}") @ResponseBody public void displayFile(@PathVariable("filename")String filename,HttpServletResponse response) { Result result = this.fileService.displayFile(filename); if(result.isSuccess()) { InputStream stream = (InputStream)result.getModel("inputStream"); try { OutputStream outputStream = response.getOutputStream(); if (stream != null) { BufferedInputStream bis = new BufferedInputStream(stream, 128); while (true) { int _current = bis.read(); if (_current < 0) { break; } outputStream.write((byte) _current); } } } catch (Exception e) { // } finally { try { stream.close(); }catch (Exception e) { // } } } } @RequestMapping("/ds/{path}/{filename}") @ResponseBody public void displayImage(@PathVariable("filename")String filename, @PathVariable("path")String path, HttpServletResponse response) { Result result = this.fileService.displayImage(filename,path); if(result.isSuccess()) { InputStream stream = (InputStream)result.getModel("inputStream"); try { OutputStream outputStream = response.getOutputStream(); if (stream != null) { BufferedInputStream bis = new BufferedInputStream(stream, 128); while (true) { int _current = bis.read(); if (_current < 0) { break; } outputStream.write((byte) _current); } } // String timestamp = (String)result.getModel("timestamp"); // if(timestamp != null) { // response.addDateHeader("Last-Modified",Long.valueOf(timestamp)); // } } catch (Exception e) { // } finally { try { stream.close(); }catch (Exception e) { // } } } } }

public interface FileService { /** * 上传文件，制定文件的格式 * @param inputStream 输入流 * @Param format 文件格式后缀 * @param uid 文件上传者 * */ public Result uploadFile(InputStream inputStream, String format, String uid); /** * 图片上传 * @param inputStream * @param returnSize 返回接近此size的url全路径 */ public Result uploadImage(ByteArrayInputStream inputStream, String uid, Integer returnSize); /** * 删除文件 * @param filename * @return */ public Result delete(String filename); /** * 获取文件 * @param filename * @return */ public Result displayFile(String filename); public Result displayImage(String filename, String path); }

@Service public class FileServiceImpl implements FileService { protected static final Logger logger = Logger.getLogger(FileService.class); @Autowired private GridFSClient gridFSClient; @Override public Result uploadFile(InputStream inputStream, String format,String uid) { Result result = new Result(); try { String filename = gridFSClient.saveFile(inputStream,format,uid); result.addModel("filename",filename); result.setSuccess(true); } catch (Exception e) { logger.error("upload file error,",e); result.setResultCode(ResultCode.SYSTEM_ERROR); result.setMessage("文件上传失败"); } return result; } @Override public Result uploadImage(ByteArrayInputStream inputStream,String uid,Integer returnSize) { Result result = new Result(); try { String filename = gridFSClient.saveImage(inputStream,uid); result.addModel("filename",filename); if(returnSize != null && returnSize > 0) { //找出符合returnSize的path for (ImageSizeEnum e : ImageSizeEnum.values()) { if (e.size >= returnSize) { result.addModel("uri",e.path + "/" + filename); } } //返回最大尺寸 result.addModel("uri",ImageSizeEnum.PIXELS_MAX.path + "/" + filename); } result.setSuccess(true); } catch (Exception e) { logger.error("upload image error,",e); result.setResultCode(ResultCode.SYSTEM_ERROR); result.setMessage("图片上传失败"); } return result; } @Override public Result delete(String filename) { Result result = new Result(); try { gridFSClient.delete(filename); result.addModel("filename",filename); result.setSuccess(true); } catch (Exception e) { logger.error("delete file error,",e); result.setResultCode(ResultCode.SYSTEM_ERROR); result.setMessage("文件删除失败"); } return result; } @Override public Result displayImage(String filename,String path) { Result result = new Result(); try { InputStream inputStream = gridFSClient.getImage(filename,path); if(inputStream != null) { result.addModel("inputStream", inputStream); result.setSuccess(true); } } catch (Exception e) { result.setResultCode(ResultCode.SYSTEM_ERROR); } return result; } @Override public Result displayFile(String filename) { Result result = new Result(); try { InputStream inputStream = gridFSClient.getFile(filename); if(inputStream != null) { result.addModel("inputStream", inputStream); result.setSuccess(true); } } catch (Exception e) { result.setResultCode(ResultCode.SYSTEM_ERROR); } return result; } }

代码见 https://github.com/javahongxi/whatsmars/tree/1.0/whatsmars-earth/whatsmars-earth-support-web

【本文地址】

公司简介

联系我们